module ObservationsIO

using ..DatesUtil: juldate
using ..ObservationsCore: Observations, ObservationRecord, add_observation!, empty_observations

"""
    read_observations(path_obs::AbstractString, suffix::AbstractString;
                      datei::Int=0, datef::Int=typemax(Int),
                      use_obsfiles::Bool=true) -> Observations

Enhanced observation reader supporting multiple formats:
1. Standard ASCII format: `rec yyyymmdd hhmmss juldate avetime conc err num`
2. obsfiles.txt list format (Fortran-style)
3. Direct file pattern matching

Parameters:
- path_obs: Directory containing observation files
- suffix: File suffix to match (e.g., "_24", ".txt")
- datei, datef: Date range filter (YYYYMMDD format)
- use_obsfiles: Whether to look for and use obsfiles.txt list
"""
function read_observations(path_obs::AbstractString, suffix::AbstractString;
                           datei::Int=0, datef::Int=typemax(Int),
                           use_obsfiles::Bool=true)
    isdir(path_obs) || error("read_observations: path not found: $(path_obs)")

    obs = empty_observations()

    # Try to read file list from obsfiles.txt if it exists and use_obsfiles is true
    files_to_read = String[]
    obsfiles_path = joinpath(path_obs, "obsfiles.txt")

    if use_obsfiles && isfile(obsfiles_path)
        files_to_read = read_obsfiles_list(obsfiles_path, path_obs)
        obs.obsfiles_list = copy(files_to_read)
    else
        # Fallback to pattern matching
        pat = endswith(suffix, ".txt") ? suffix : (suffix * ".txt")
        files_to_read = filter(f -> endswith(lowercase(f), lowercase(pat)),
                              readdir(path_obs; join=true))
    end

    @info "Reading $(length(files_to_read)) observation files from $path_obs"

    # Process each file
    for filepath in files_to_read
        try
            read_observation_file!(obs, filepath, datei, datef)
        catch e
            @warn "Failed to read observation file: $filepath" exception=e
        end
    end

    @info "Loaded $(length(obs)) observations from $(length(obs.by_station)) stations"

    return obs
end

"""
    read_obsfiles_list(obsfiles_path::String, base_path::String) -> Vector{String}

Read the obsfiles.txt format list. Format is:
Line 1: Number of files
Lines 2-N: Filenames (relative to base_path)
"""
function read_obsfiles_list(obsfiles_path::String, base_path::String)
    files = String[]

    open(obsfiles_path, "r") do io
        # Read number of files
        first_line = strip(readline(io))
        nfiles = parse(Int, first_line)

        # Read file names
        for _ in 1:nfiles
            line = strip(readline(io))
            if !isempty(line)
                # Make full path
                full_path = joinpath(base_path, line)
                if isfile(full_path)
                    push!(files, full_path)
                else
                    @warn "File listed in obsfiles.txt not found: $full_path"
                end
            end
        end
    end

    return files
end

"""
    read_observation_file!(obs::Observations, filepath::String,
                          datei::Int, datef::Int)

Read a single observation file and add records to the Observations structure.
Supports multiple formats and handles various ASCII layouts.
"""
function read_observation_file!(obs::Observations, filepath::String,
                               datei::Int, datef::Int)
    filename = basename(filepath)

    open(filepath, "r") do io
        lines = readlines(io)
        if isempty(lines)
            @warn "Empty observation file: $filepath"
            return
        end

        # Detect format and header
        start_line = 1
        header_line = strip(lines[1])

        # Check if first line is a header
        if is_header_line(String(header_line))
            start_line = 2
        end

        # Process data lines
        for (line_num, line) in enumerate(lines[start_line:end])
            s = strip(line)
            isempty(s) && continue

            try
                record = parse_observation_line(String(s), line_num + start_line - 1, filepath)
                if record !== nothing
                    # Apply date filter
                    if (record.yyyymmdd >= datei) && (record.yyyymmdd <= datef)
                        add_observation!(obs, record; source_file=filename)
                    end
                end
            catch e
                @warn "Failed to parse line $(line_num + start_line - 1) in $filepath: $s" exception=e
            end
        end
    end
end

"""
    is_header_line(line::String) -> Bool

Detect if a line is a header line.
"""
function is_header_line(line::AbstractString)
    line_lower = lowercase(line)
    # Look for header keywords
    header_keywords = ["rec", "yyyymmdd", "hhmmss", "juldate", "avetime", "conc", "err", "num"]
    count = sum(occursin(keyword, line_lower) for keyword in header_keywords)
    return count >= 4  # If at least 4 keywords present, likely a header
end

"""
    parse_observation_line(line::String, line_num::Int, filepath::String) -> Union{ObservationRecord, Nothing}

Parse a single observation data line into an ObservationRecord.
Supports flexible column parsing and various formats.
"""
function parse_observation_line(line::AbstractString, line_num::Int, filepath::String)
    parts = split(line)

    # Need at least 8 columns for standard format
    if length(parts) < 8
        @debug "Line $line_num in $filepath has only $(length(parts)) columns, skipping"
        return nothing
    end

    try
        # Standard format: rec yyyymmdd hhmmss juldate avetime conc err num
        rec = String(parts[1])
        ymd = parse(Int, parts[2])
        hms = parse(Int, parts[3])

        # Handle julian date - prefer provided value, compute if invalid
        jdat = try
            val = parse(Float64, parts[4])
            # Basic sanity check for julian date (should be reasonable)
            if val > 1000000 && val < 3000000
                val
            else
                juldate(ymd, hms)
            end
        catch
            juldate(ymd, hms)
        end

        avt = parse(Float64, parts[5])
        conc = parse(Float64, parts[6])
        err = parse(Float64, parts[7])
        num = parse(Int, parts[8])

        # Basic data validation
        if conc <= -999.0  # Common missing data flag
            @debug "Skipping observation with missing concentration flag: $conc"
            return nothing
        end

        return ObservationRecord(
            rec=rec, yyyymmdd=ymd, hhmmss=hms, jdate=jdat,
            avetime=avt, conc=conc, err=err, num=num
        )

    catch e
        @warn "Failed to parse observation line $line_num in $filepath: $line" exception=e
        return nothing
    end
end

"""
    read_observations_legacy(path_obs::AbstractString, suffix::AbstractString;
                            datei::Int=0, datef::Int=typemax(Int)) -> Vector{ObservationRecord}

Legacy function that returns a vector of ObservationRecord for backward compatibility.
"""
function read_observations_legacy(path_obs::AbstractString, suffix::AbstractString;
                                 datei::Int=0, datef::Int=typemax(Int))
    obs = read_observations(path_obs, suffix; datei=datei, datef=datef, use_obsfiles=false)

    # Convert to legacy format
    records = ObservationRecord[]
    for i in 1:length(obs)
        record = ObservationRecord(
            rec=obs.stations[i],
            yyyymmdd=Int(obs.jdates[i] * 1e8) ÷ 1000000,  # Approximate conversion
            hhmmss=0,  # Would need to parse from datetime
            jdate=obs.jdates[i],
            avetime=obs.avetime[i],
            conc=obs.concentrations[i],
            err=obs.measurement_errors[i],
            num=1
        )
        push!(records, record)
    end

    return records
end

end # module

